Project 4 - Advanced Lane Finding

Steps:

  1. Camera calibration - Done
  2. Distortion correction - Done
  3. Color/gradient threshold - Done
  4. Perspective transform - Done
  5. Detect lane lines - Done
  6. Determine the lane curvature - Done

Initial Imports

In [24]:
# Libraries used throughout are imported here

import pickle
import cv2
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import matplotlib.image as mpimg
import glob

# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML

Useful Methods

In [51]:
#Most methods that need to be re-used are defined here 

# Function that takes an image, object points, and image points
# performs the camera calibration, image distortion correction and
# returns the undistorted image
def cal_undistort(img, objpoints, imgpoints):
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    # Use cv2.calibrateCamera and cv2.undistort()
    ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1],None,None)
    # Undistorting a test image:
    undist = cv2.undistort(img, mtx, dist, None, mtx)
    return undist

def corners_unwarp(img, corners, nx, ny, mtx, dist):
#     # Use the OpenCV undistort() function to remove distortion
#     undist = cv2.undistort(img, mtx, dist, None, mtx)
#     # Convert undistorted image to grayscale
#     gray = cv2.cvtColor(undist, cv2.COLOR_BGR2GRAY)
    offset = 100 # offset for dst points
    # Grab the image shape
    img_size = (img.shape[1], img.shape[0])
    src = np.float32([corners[0], corners[1], corners[-1], corners[-nx]])
    # For destination points, I'm arbitrarily choosing some points to be
    # a nice fit for displaying our warped result 
    # again, not exact, but close enough for our purposes
    dst = np.float32([[offset, offset], [img_size[0]-offset, offset], 
                                     [img_size[0]-offset, img_size[1]-offset], 
                                     [offset, img_size[1]-offset]])
    # Given src and dst points, calculate the perspective transform matrix
    M = cv2.getPerspectiveTransform(src, dst)
    # Warp the image using OpenCV warpPerspective()
    warped = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_LINEAR)

    # Return the resulting image and matrix
    return warped, M

# Function that takes an image, gradient orientation,
# and threshold min / max values and returns output with threshold applied
def abs_sobel_thresh(img, orient='x', sobel_kernel=3, thresh=(0,255)):
    # Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Apply x or y gradient with the OpenCV Sobel() function
    # and take the absolute value
    if orient == 'x':
        abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 1, 0))
    if orient == 'y':
        abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 0, 1))
    # Rescale back to 8 bit integer
    scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel))
    # Create a copy and apply the threshold
    binary_output = np.zeros_like(scaled_sobel)
    # Here I'm using inclusive (>=, <=) thresholds, but exclusive is ok too
    binary_output[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1

    # Return the result
    return binary_output

# Function that takes image, kernel size, and threshold and returns
# magnitude of the gradient
def mag_threshold(img, sobel_kernel=3, mag_thresh=(0, 255)):
    # Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Take both Sobel x and y gradients
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # Calculate the gradient magnitude
    gradmag = np.sqrt(sobelx**2 + sobely**2)
    # Rescale to 8 bit
    scale_factor = np.max(gradmag)/255
    gradmag = (gradmag/scale_factor).astype(np.uint8)
    # Create a binary image of ones where threshold is met, zeros otherwise
    binary_output = np.zeros_like(gradmag)
    binary_output[(gradmag >= mag_thresh[0]) & (gradmag <= mag_thresh[1])] = 1

    # Return the binary image
    return binary_output

#function to threshold HSV color spectrum in an image for a given range
def color_threshold_hsv(img, channel="s", thresh=(170,255)):
    img = np.copy(img)
    # Convert to HSV color space and separate the V channel
    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HLS).astype(np.float)
    h_channel = hsv[:,:,0]
    l_channel = hsv[:,:,1]
    s_channel = hsv[:,:,2]
    
    if channel == "h":
        target_channel = h_channel
    elif channel == "l":
        target_channel = l_channel
    else:
        target_channel = s_channel
    
    # Threshold color channel
    binary_output = np.zeros_like(target_channel)
    binary_output[(target_channel >= thresh[0]) & (target_channel <= thresh[1])] = 1
    
    return binary_output

#function to threshold RGB color spectrum in an image for a given range
def color_threshold_rgb(img, channel="r", thresh=(170,255)):
    img = np.copy(img)
    r_channel = img[:,:,0]
    g_channel = img[:,:,1]
    b_channel = img[:,:,2]
    
    if channel == "r":
        target_channel = r_channel
    elif channel == "g":
        target_channel = g_channel
    else:
        target_channel = b_channel
    
    # Threshold color channel
    binary_output = np.zeros_like(target_channel)
    binary_output[(target_channel >= thresh[0]) & (target_channel <= thresh[1])] = 1
    
    return binary_output
    
#function to threshold HSV color spectrum in an image for a given range
def color_threshold_yuv(img, channel="v", thresh=(0,255)):
    img = np.copy(img)
    # Convert to YUV color space and separate the V channel
    yuv = cv2.cvtColor(img, cv2.COLOR_RGB2YUV).astype(np.float)
    y_channel = yuv[:,:,0]
    u_channel = yuv[:,:,1]
    v_channel = yuv[:,:,2]
    
    if channel == "y":
        target_channel = y_channel
    elif channel == "u":
        target_channel = u_channel
    else:
        target_channel = v_channel
    
    # Threshold color channel
    binary_output = np.zeros_like(target_channel)
    binary_output[(target_channel >= thresh[0]) & (target_channel <= thresh[1])] = 1
    
    return binary_output

# Function to threshold gradient direction in an image for a given range and Sobel kernel
def dir_threshold(img, sobel_kernel=3, thresh=(0, np.pi/2)):
    # Grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Calculate the x and y gradients
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # Take the absolute value of the gradient direction,
    # apply a threshold, and create a binary image result
    absgraddir = np.arctan2(np.absolute(sobely), np.absolute(sobelx))
    binary_output =  np.zeros_like(absgraddir)
    binary_output[(absgraddir >= thresh[0]) & (absgraddir <= thresh[1])] = 1

    # Return the binary image
    return binary_output

def pipeline_hsv(img, sobel_kernel=3, s_thresh=(170, 255), sx_thresh=(20, 100), sy_thresh=(20,120), mag_thresh=(0,255)):
    img = np.copy(img)
    # Convert to HSV color space and separate the V channel
    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HLS).astype(np.float)
    l_channel = hsv[:,:,1]
    s_channel = hsv[:,:,2]
    
    # Sobel x
    sobelx = cv2.Sobel(l_channel, cv2.CV_64F, 1, 0, ksize=23) # Take the derivative in x
    abs_sobelx = np.absolute(sobelx) # Absolute x derivative to accentuate lines away from horizontal
    #scaled_sobel = np.uint8(255*abs_sobelx/np.max(abs_sobelx))
    
    # Sobel y
    sobely = cv2.Sobel(s_channel, cv2.CV_64F, 0, 1, ksize=23) # Take the derivative in y
    abs_sobely = np.absolute(sobely) # Absolute y derivative to accentuate lines away from vertical
    scaled_sobel = np.uint8(255*abs_sobely/np.max(abs_sobely))
    
    # Threshold x gradient
    sxbinary = np.zeros_like(scaled_sobel)
    sxbinary[(scaled_sobel >= sx_thresh[0]) & (scaled_sobel <= sx_thresh[1])] = 1
    
    # Threshold y gradient
    sybinary = np.zeros_like(scaled_sobel)
    sybinary[(scaled_sobel >= sy_thresh[0]) & (scaled_sobel <= sy_thresh[1])] = 1
    
    magbinary = mag_threshold(img, sobel_kernel, mag_thresh)
    
    # Threshold color channel
    s_binary = np.zeros_like(s_channel)
    s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1
    # Stack each channel
    # Note color_binary[:, :, 0] is all 0s, effectively an all black image. It might
    # be beneficial to replace this channel with something else.
    color_binary = np.dstack(( np.zeros_like(sxbinary), magbinary, sxbinary, sybinary, s_binary))
    return color_binary

def myPipeline(img):
    ksize = 3
    gradx = abs_sobel_thresh(img, orient='x', sobel_kernel=ksize, thresh=(10, 100))
    grady = abs_sobel_thresh(img, orient='y', sobel_kernel=ksize, thresh=(5, 250))
    mag_binary = mag_threshold(img, sobel_kernel=ksize, mag_thresh=(5, 100))
    dir_binary = dir_threshold(img, sobel_kernel=ksize, thresh=(0, np.pi/2))
    s_binary = color_threshold_hsv(img, "s", (120,255))
    v_binary = color_threshold_yuv(img,"v", (0,105))
    r_binary = color_threshold_rgb(img,"r", (230,255))
    result = np.zeros_like(dir_binary)
    result[((gradx == 1) & (grady == 1)) | ((mag_binary == 1) & (dir_binary == 1)) & ((s_binary == 1)) | ((v_binary ==1) | (r_binary == 1))] = 1
    return result

def warp(img):
    img_size = (img.shape[1],img.shape[0])
    #Four Source coordinates
    source = np.float32([[(.42 * img.shape[1], .65 * img.shape[0] ),(.58 * img.shape[1], .65 * img.shape[0]),(0 * img.shape[1],img.shape[0]),(1 * img.shape[1], img.shape[0])]])
    #vertices = np.array([[(.16 * imshape[1], .92 * imshape[0]), (.45 * imshape[1], .63 * imshape[0] ),(.6 * imshape[1], .63 * imshape[0] ),(.9 * imshape[1],.92 *imshape[0])]], dtype=np.int32)  

    #Four desired coordinates
    dest = np.float32([[0,0],[img.shape[1],0],[0,img.shape[0]],[img.shape[1],img.shape[0]]])
    M = cv2.getPerspectiveTransform(source, dest)
    Minv = cv2.getPerspectiveTransform(dest, source)
    warped = cv2.warpPerspective(img,M,img_size)
    return warped, M, Minv

# Define a class to receive the characteristics of each line detection
class Line():
    def __init__(self):
        # was the line detected in the last iteration?
        self.detected = False  
        # x values of the last n fits of the line
        self.recent_xfitted = [] 
        #average x values of the fitted line over the last n iterations
        self.bestx = None     
        #polynomial coefficients averaged over the last n iterations
        self.best_fit = None  
        #polynomial coefficients for the most recent fit
        self.current_fit = [np.array([False])]  
        #radius of curvature of the line in some units
        self.radius_of_curvature = None 
        #distance in meters of vehicle center from the line
        self.line_base_pos = None 
        #difference in fit coefficients between last and new fits
        self.diffs = np.array([0,0,0], dtype='float') 
        #x values for detected line pixels
        self.allx = None  
        #y values for detected line pixels
        self.ally = None

# methods from Project 1:
# -----------------------
##Custom Display Image helper function
def displayImage(image, gray=False, title="Displayed Image"):
    """loads the image and displays in output, in grayscale if param gray is true
    """
    if(not gray):
        plt.imshow(image)
    #call as plt.imshow(gray, cmap='gray') to show a grayscaled image
    else:
        plt.imshow(image, cmap='gray')
        
def grayscale(img):
    """Applies the Grayscale transform
    This will return an image with only one color channel
    but NOTE: to see the returned image as grayscale
    you should call plt.imshow(gray, cmap='gray')"""
    return cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)

def region_of_interest(img, vertices):
    """
    Applies an image mask.
    
    Only keeps the region of the image defined by the polygon
    formed from `vertices`. The rest of the image is set to black.
    """
    #defining a blank mask to start with
    mask = np.zeros_like(img)   
    
    #defining a 3 channel or 1 channel color to fill the mask with depending on the input image
    if len(img.shape) > 2:
        channel_count = img.shape[2]  # i.e. 3 or 4 depending on your image
        ignore_mask_color = (255,) * channel_count
    else:
        ignore_mask_color = 255
        
    #filling pixels inside the polygon defined by "vertices" with the fill color    
    cv2.fillPoly(mask, vertices, ignore_mask_color)
    
    #returning the image only where mask pixels are nonzero
    masked_image = cv2.bitwise_and(img, mask)
    return masked_image

def img_overlay(img_lines, initial_img, α=0.8, β=1., λ=0.):
    """
    `img_lines` is the output of lane dectection, An image with lines drawn on it.
    Should be a blank image (all black) with lines drawn on it.
    
    `initial_img` should be the image before any processing.
    
    The result image is computed as follows:
    
    initial_img * α + img * β + λ
    NOTE: initial_img and img must be the same shape!
    """
    return cv2.addWeighted(initial_img, α, img, β, λ)

# Other Helpers -------->
def readImage(img):
    """
    Read the image and convert to RGB for plot
    """
    img_read = cv2.imread(img)
    img_RGB =  cv2.cvtColor(img_read, cv2.COLOR_BGR2RGB)
    return img_RGB
#Ignore the .DS_Store files because of Mac OS
def listdir_nohidden(path):
    return glob.glob(os.path.join(path, '*'))

Implementation

First I add the image files from the directory and inspect to see what the size is and that the plotting works correctly

In [26]:
#Import images
import os

calImageFiles = listdir_nohidden("camera_cal")
calImages = []
testImageFiles = listdir_nohidden("test_images")
testImages = []
for f in calImageFiles:
    calImages.append(readImage(f))

for imageText in testImageFiles:
    testImages.append(readImage(imageText))
In [27]:
img = testImages[0]
#printing out some stats and plotting
print('This image is:', type(img), 'with dimesions:', img.shape)
displayImage(img, False)
This image is: <class 'numpy.ndarray'> with dimesions: (720, 1280, 3)

Find Corners for Image Points and Object Points to Calibrate and Undistort

My next step is to calibrate the camera and undistort based upon the checkerboard images provided

In [28]:
# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((6*9,3), np.float32)
objp[:,:2] = np.mgrid[0:9, 0:6].T.reshape(-1,2)

# Arrays to store object points and image points from all the images.
objpoints = [] # 3d points in real world space
imgpoints = [] # 2d points in image plane.

# Step through the list and search for corners
for idx, fname in enumerate(calImageFiles):
    img = cv2.imread(fname)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # find the corners
    #corners = cv2.goodFeaturesToTrack(gray,nCorners,0.01,10)
    ret,corners = cv2.findChessboardCorners(gray, (9,6), None)
    # If found, add object points, image points
    if ret == True:
        objpoints.append(objp)
        imgpoints.append(corners)
        # Draw and display the corners
        cv2.drawChessboardCorners(img, (9,6), corners, ret)
        #write_name = 'corners_found'+str(idx)+'.jpg'
        #cv2.imwrite(write_name, img)
In [29]:
#Test on first calibration image
distImg = calImages[0]
undistorted = cal_undistort(distImg, objpoints, imgpoints)
cv2.imwrite('output_images/undistorted_checkerboard.jpg', undistorted)
plt.imshow(undistorted), plt.show()
Out[29]:
(<matplotlib.image.AxesImage at 0x126ff7f60>, None)

With image and object points perform calibrate and undistort

Now with the calibration completed I can undistort the images from the car and visualize below

In [35]:
undistortedImages = []
for idx,image in enumerate(testImages):
    undistorted = cal_undistort(image, objpoints, imgpoints)
    cv2.imwrite('output_images/undistortedTestImage'+str(idx)+'.jpg', cv2.cvtColor(undistorted, cv2.COLOR_BGR2RGB))
    undistortedImages.append(undistorted)

Visualize undistortion

In [31]:
for idx, img in enumerate(undistortedImages):
    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
    ax1.imshow(testImages[idx])
    ax1.set_title('Original Image', fontsize=30)
    ax2.imshow(img)
    ax2.set_title('Undistorted Image', fontsize=30)

Determine region of interest

The next step is to determine the area of perspective transformation on the road. I chose a wide area rather than a narrow band along side the lines. I chose to do this because I had planned on gathering more information about the road edges in the future or if there was a case where lanes were wider or no line was available on a particular side.

In [32]:
# Determine the region of interest
img = np.copy(undistortedImages[0])
imshape = img.shape
pts = np.array([(0 * imshape[1], imshape[0]), (.41 * imshape[1], .65 * imshape[0] ),(.59 * imshape[1], .65 * imshape[0] ),(imshape[1], imshape[0])], np.int32)
pts = pts.reshape((-1,1,2))
img = cv2.polylines(img,[pts],True,(255,0,0))
# vertices = np.array([[(0 * imshape[1], imshape[0]), (.41 * imshape[1], .65 * imshape[0] ),(.59 * imshape[1], .65 * imshape[0] ),(imshape[1], imshape[0])]], dtype=np.int32)  
# masked_image = region_of_interest(img, vertices)
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()

ax1.imshow(undistortedImages[0])
ax1.set_title('Original Undistorted Image', fontsize=40)

ax2.imshow(img)
ax2.set_title('Area of transformation', fontsize=40)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Perform the Perspective Transformation and visualize

In [38]:
top_down_images = []
for idx,img in enumerate(undistortedImages):
    top_down, M, Minv = warp(img)
    unwarped = cv2.warpPerspective(top_down, Minv,(undistortedImages[0].shape[1],undistortedImages[0].shape[0]),flags=cv2.INTER_LINEAR)
    top_down_images.append(top_down)
    cv2.imwrite('output_images/top_down_transformation'+str(idx)+'.jpg', cv2.cvtColor(top_down, cv2.COLOR_BGR2RGB))
    # Plot the result
    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
    f.tight_layout()

    ax1.imshow(img)
    ax1.set_title('Original Undistorted Image', fontsize=40)

    ax2.imshow(top_down)
    ax2.set_title('Top_Down Result', fontsize=40)
    plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Apply Color and Gradient thresholds

The next step is apply color and gradient thresholds in my pipeline to determine the pixels most likely to be lane lines. My pipeline consists of color and Sobel gradient thresholds.

In [48]:
edges = []
for idx, img in enumerate(top_down_images):
    result = myPipeline(img) #mag_threshold(img, 3, (10,200)) #abs_sobel_thresh(img, orient='x', sobel_kernel=3, thresh=(0,255)):
    
    #add to edges
    edges.append(result)
    # Plot the result
    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
    f.tight_layout()
    
    ax1.imshow(img)
    ax1.set_title('Warped Image', fontsize=40)

    ax2.imshow(result, cmap='gray')
    ax2.set_title('Pipeline Result', fontsize=40)
    plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
    f.savefig('output_images/pipeline_result'+str(idx)+'.jpg')

Detect Lines

The next step is to determine where the lane lines are and separate them from the noise after the thresholds are applied in the pipeline. Once the points of the line are determined in the image I then fit a second-order polynomial line to best fit those points.

In [23]:
from scipy import signal

def detectLines(img, line, direction="left"):
    '''
     # was the line detected in the last iteration?
        self.detected = False  
        # x values of the last n fits of the line
        self.recent_xfitted = [] 
        #average x values of the fitted line over the last n iterations
        self.bestx = None     
        #polynomial coefficients averaged over the last n iterations
        self.best_fit = None  
        #polynomial coefficients for the most recent fit
        self.current_fit = [np.array([False])]  
        #radius of curvature of the line in some units
        self.radius_of_curvature = None 
        #distance in meters of vehicle center from the line
        self.line_base_pos = None 
        #difference in fit coefficients between last and new fits
        self.diffs = np.array([0,0,0], dtype='float') 
        #x values for detected line pixels
        self.allx = None  
        #y values for detected line pixels
        self.ally = None
    '''
    
    winWidth = 25
    winHeight = 50 #bigger numbers create smaller windows as it is calculated as a proportion to the img height

    if not line.detected:
        histogram = np.sum(img[img.shape[0]*(.5):,0:img.shape[1]], axis=0)
        #plt.plot(histogram)

        #find two peaks first is the left and last is the right
        #initial peak
        peakind = signal.find_peaks_cwt(histogram, np.arange(100,200))
        if direction == 'left':
            peak = peakind[0]
        else:
            peak = peakind[-1]
        
        #move the sliding window across and gather the points
        yvals = []
        xvals = []
        
        for i in range(winHeight):
            #peaks may be at the edge so we need to stop at the edge
            if direction == 'left':
                if peak < winWidth:
                    peak = winWidth
            else:
                if peak >= (img.shape[1] - winWidth):
                    peak = img.shape[1] - winWidth - 1
            
            for yval in range(int(img.shape[0]*((winHeight-i-1)/winHeight)), int(img.shape[0]*((winHeight-i)/winHeight))):
                for xval in range(peak-winWidth, peak+winWidth):
                    if img[yval][xval] == 1.0:
                        yvals.append(yval)
                        xvals.append(xval)
            #find new peaks to move the window accordingly for next iteration
            #new peaks will be the max in the current window plus the beginning of the window...
            
            histogram = np.sum(img[img.shape[0]*((winHeight-i-1)/winHeight):img.shape[0]*((winHeight-i)/winHeight),peak-winWidth:peak+winWidth], axis=0)
            if len(signal.find_peaks_cwt(histogram, np.arange(100,200))) > 0:
                peak = np.amax(signal.find_peaks_cwt(histogram, np.arange(100,200))) + (peak-winWidth)
            else: #look in bigger window
                winWidthBig = 100
                histogram = np.sum(img[img.shape[0]*((winHeight-i-1)/winHeight):img.shape[0]*((winHeight-i)/winHeight),peak-winWidthBig:peak+winWidthBig], axis=0)
                if len(histogram > 0):
                    if len(signal.find_peaks_cwt(histogram, np.arange(100,200))) > 0:
                        peak = np.amax(signal.find_peaks_cwt(histogram, np.arange(100,200))) + (peak-winWidthBig)

        yvals = np.asarray(yvals)
        xvals = np.asarray(xvals)
       
        line.allx = xvals
        line.ally = yvals
        
        # Fit a second order polynomial to lane line
        fit = np.polyfit(yvals, xvals, 2)
        
        line.current_fit = fit
        line.best_fit = fit
        #print(fit)
        
        fitx = fit[0]*yvals**2 + fit[1]*yvals + fit[2]
        #print(fitx)
        
        line.recent_xfitted.append(fitx)
        line.bestx = fitx
        
        # Set detected to true - This is now done when the line is checked for accuracy
        #line.detected = True 
        
    else:
        #initial peak - use previous line x
        peak = line.bestx[0]
        prev_line = copy(line)
        
        #move the sliding window across and gather the points
        yvals = []
        xvals = []
        
        for i in range(winHeight):
            #peaks may be at the edge so we need to stop at the edge
            if direction == 'left':
                if int(peak) < winWidth:
                    peak = winWidth
            else:
                if int(peak) >= (img.shape[1] - winWidth):
                    peak = img.shape[1] - winWidth - 1
                    
            for yval in range(int(img.shape[0]*((winHeight-i-1)/winHeight)), int(img.shape[0]*((winHeight-i)/winHeight))):
                for xval in range(int(peak-winWidth), int(peak+winWidth)):
                    if img[yval][xval] == 1.0:
                        yvals.append(yval)
                        xvals.append(xval)
            #use bestx to keep going over the line
            peak = line.bestx[(i + 1)%len(line.bestx)]

        yvals = np.asarray(yvals)
        xvals = np.asarray(xvals)
        
        line.allx = xvals
        line.ally = yvals
        
        # Fit a second order polynomial to lane line
        fit = np.polyfit(yvals, xvals, 2)
        line.current_fit = fit
        fitx = fit[0]*yvals**2 + fit[1]*yvals + fit[2]
        
        isOk = checkDetection(prev_line, line)
        if isOk:
            if len(line.recent_xfitted) > 10:
                #remove the first element
                line.recent_xfitted.pop(0)
                line.recent_xfitted.append(fitx)
                line.bestx = fitx
                line.best_fit = fit
        else:
            #line lost, go back to sliding window
            line.detected = false
        
    return line

def checkDetection(prev_line, next_line):
    # Checking that they have similar curvature
    left_curvature = getCurvature(prev_line.allx, prev_line.ally, prev_line.current_fit )
    right_curvature = getCurvature(next_line.allx, next_line.ally, next_line.current_fit)
    # Checking that they are separated by approximately the right distance horizontally
    left_x = prev_line.recent_xfitted[0][0]
    right_x = next_line.recent_xfitted[0][0]
    if (np.absolute(left_x - right_x) > 1000) | (np.absolute(left_curvature - right_curvature) > 100): #in pixels, not meters
        prev_line.detected = False
        next_line.detected = False
        return False
    # Checking that they are roughly parallel- TODO // the curvature tells this partially...
    
    prev_line.detected = True #in case these are different lines that are being compared
    next_line.detected = True
    return True
    

#to keep track of lines
detectedLines = []
for idx,img in enumerate(edges):
    left_line = Line()
    right_line = Line()
    left_line = detectLines(img, left_line, 'left')
    right_line = detectLines(img, right_line, 'right')
    detectedLines.append({'left_line': left_line, 'right_line': right_line })
    
    # Plot the result
    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
    f.tight_layout()
    
    ax1.imshow(img, cmap='gray')
    ax1.set_title('Pipeline Results', fontsize=40)

    # Plot up the data
    ax2.plot(left_line.allx, left_line.ally, 'o', color='red')
    ax2.plot(right_line.allx, right_line.ally, 'o', color='blue')
    plt.xlim(0, 1280)
    plt.ylim(0, 720)
    ax2.plot(left_line.bestx, left_line.ally, color='green', linewidth=3)
    ax2.plot(right_line.bestx, right_line.ally, color='green', linewidth=3)
    plt.gca().invert_yaxis() # to visualize as we do the images
    ax2.set_title('Line Fits', fontsize=40)
    plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
    f.savefig('output_images/line_fit_'+str(idx)+'.jpg')

    

Determine the lane and project back onto the undistorted Image

The next step is to use the two fitted lines for the lane and create a polygon fill for the lane and project it back into the original undistorted image using the inverse transformation matrix from the transformation process earlier.

In [50]:
# Define y-value where we want radius of curvature
# I'll choose the maximum y-value, corresponding to the bottom of the image
def getCurvature(line_x, line_y, fit):
    
    y_eval = np.max(line_y)
    curverad = ((1 + (2*fit[0]*y_eval + fit[1])**2)**1.5) \
                                 /np.absolute(2*fit[0])
    
    # Define conversions in x and y from pixels space to meters
    ym_per_pix = 30/720 # meters per pixel in y dimension
    xm_per_pix = 3.7/700 # meteres per pixel in x dimension

    fit_cr = np.polyfit(line_y*ym_per_pix, line_x*xm_per_pix, 2)
    
    curverad = ((1 + (2*fit_cr[0]*y_eval + fit_cr[1])**2)**1.5) \
                                 /np.absolute(2*fit_cr[0])
    
    # Now our radius of curvature is in meters
    #print(left_curverad, 'm', right_curverad, 'm')
    # Example values: 3380.7 m    3189.3 m

    return curverad
    # Example values: 1163.9    1213.7
    
def getCenterDiff(img,lines):
    #midpoint of the lines (half the polyfill width)
    midPoly = (lines['right_line'].bestx[0] - lines['left_line'].bestx[0]) / 2
    #midpoint of the image (half the image length)
    midImage = img.shape[0] / 2
    
    diffInPix = midImage - midPoly
    #convert to meters
    xm_per_pix = 3.7/700 # meteres per pixel in x dimension
    result = diffInPix * xm_per_pix
    lines['left_line'].line_base_pos = result
    lines['right_line'].line_base_pos = result
    return result
    

def imgWithLane(undist, warped, lines):
    undist = np.copy(undist)
    # Create an image to draw the lines on
    warp_zero = np.zeros_like(img).astype(np.uint8)
    color_warp = np.dstack((warp_zero, warp_zero, warp_zero))

    # Recast the x and y points into usable format for cv2.fillPoly()
    pts_left = np.array([np.transpose(np.vstack([lines['left_line'].allx, lines['left_line'].ally]))])
    pts_right = np.array([np.flipud(np.transpose(np.vstack([lines['right_line'].allx, lines['right_line'].ally])))])
    pts = np.hstack((pts_left, pts_right))
    
     # Draw the lane onto the warped blank image
    cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))
    
    #Draw the points on the image
    for idx,pt in enumerate(lines['left_line'].ally):
        #cv2.circle(img,(447,63), 63, (0,0,255), -1)
        cv2.circle(color_warp,(lines['left_line'].allx[idx], pt), 2, (255,0,0), -1)
    
    for idx,pt in enumerate(lines['right_line'].ally):
        #cv2.circle(img,(447,63), 63, (0,0,255), -1)
        cv2.circle(color_warp,(lines['right_line'].allx[idx], pt), 2, (0,0,255), -1)
    
    #get the radius curvature
    left_curverad = getCurvature(lines['left_line'].allx, lines['left_line'].ally, lines['left_line'].best_fit)
    right_curverad = getCurvature(lines['right_line'].allx, lines['right_line'].ally, lines['right_line'].best_fit)
    left_text = 'Left Curvature Radius: ' + str(np.around(left_curverad,2)) + 'm'
    right_text = 'Right Curvature Radius: ' + str(np.around(right_curverad,2)) + 'm'
    
    #get the distance from the center
    center_diff = getCenterDiff(undist, lines)
    if center_diff < 0:
        center_diff_text = 'Vehicle Position: ' + str(np.around(np.absolute(center_diff),2)) + 'm left of center'
    else:
        center_diff_text = 'Vehicle Position: ' + str(np.around(center_diff,2)) + 'm right of center'
        
    font = cv2.FONT_HERSHEY_SIMPLEX
    cv2.putText(undist,left_text,(10,50), font, 1,(255,255,255),2,cv2.LINE_AA)
    cv2.putText(undist,right_text,(10,100), font, 1,(255,255,255),2,cv2.LINE_AA)
    cv2.putText(undist,center_diff_text,(10,150), font, 1,(255,255,255),2,cv2.LINE_AA)

    # Warp the blank back to original image space using inverse perspective matrix (Minv)
    newwarp = cv2.warpPerspective(color_warp, Minv, (img.shape[1], img.shape[0])) 
    # Combine the result with the original image
    result = cv2.addWeighted(undist, 1, newwarp, 0.3, 0)
    return result

#check test images
for idx,img in enumerate(edges):
    result = imgWithLane(undistortedImages[idx],img, detectedLines[idx])
    cv2.imwrite('output_images/imgWithLane'+str(idx)+'.jpg', cv2.cvtColor(result, cv2.COLOR_BGR2RGB))
    # Plot the result
    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
    f.tight_layout()
    
    ax1.imshow(undistortedImages[idx], cmap='gray')
    ax1.set_title('Undistorted Image', fontsize=40)
    
    ax2.imshow(result)
    ax2.set_title('Image with lane detected', fontsize=40)

Test performance on video

Now with the images performing well it's time to test on a video!

In [19]:
#new lines to keep track in video
theLeftLine = Line()
theRightLine = Line()

def processImage(img):
    #go through each step in the process
    #1. Calibrate and undistort
    undistorted = cal_undistort(img, objpoints, imgpoints)
    #2. warp
    top_down, M, Minv = warp(undistorted)
    #3. Convert to binary with thresholds
    binary_top_down = myPipeline(top_down)
    #4. Detect the lines and fit polynomial
    leftline = detectLines(binary_top_down, theLeftLine, 'left')
    rightline = detectLines(binary_top_down, theRightLine, 'right')
    lines = {'left_line': leftline, 'right_line': rightline }
    #5. Unwarp and project onto original image
    result = imgWithLane(undistorted, binary_top_down, lines)
    return result
    
    

The video ouput

In [338]:
video_output = 'project_video_output.mp4'
clip1 = VideoFileClip("project_video.mp4")

output_clip = clip1.fl_image(processImage) #NOTE: this function expects color images!!
%time output_clip.write_videofile(video_output, audio=False)
[MoviePy] >>>> Building video project_video_output.mp4
[MoviePy] Writing video project_video_output.mp4
100%|█████████▉| 1260/1261 [5:27:29<00:04,  4.03s/it]     
[MoviePy] Done.
[MoviePy] >>>> Video ready: project_video_output.mp4 

CPU times: user 1h 42min 29s, sys: 5min 57s, total: 1h 48min 26s
Wall time: 5h 27min 30s
In [339]:
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(video_output))
Out[339]:

Challenge Video 1

I did the challenge video and performed just ok. The problem my lane detecter had was with the lane width. I grabbed a wide area for the perspective transformation and these lanes are more narrow than the other video the result is that the edge of the road gets confused as a lane line.

In [357]:
challenge_video_output = 'challenge_video_output.mp4'
clip2 = VideoFileClip("challenge_video.mp4")

output_clip_challenge = clip2.fl_image(processImage) #NOTE: this function expects color images!!
%time output_clip_challenge.write_videofile(challenge_video_output, audio=False)
[MoviePy] >>>> Building video challenge_video_output.mp4
[MoviePy] Writing video challenge_video_output.mp4
100%|██████████| 485/485 [33:37<00:00,  4.34s/it]
[MoviePy] Done.
[MoviePy] >>>> Video ready: challenge_video_output.mp4 

CPU times: user 33min 36s, sys: 38.2 s, total: 34min 14s
Wall time: 33min 38s
In [358]:
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(challenge_video_output))
Out[358]:

Challenge on Harder Video

I also looked at the harder video challenge to see performance there. As before the result with the narrow lanes and the road edge was not good. Also the motorcycle at the beginning gets confused with lane lines so the check on the lane line consistency failed here. I ran out of time to run more thorough tests and make improvements but I plan on going back and working on it some more to get better results!

NOTE: There was an error in the code that occurred while writing the video output. I ran out of time to correct it but plan on it correcting it soon.

In [26]:
harder_challenge_video_output = 'harder_challenge_video_output.mp4'
clip3 = VideoFileClip("harder_challenge_video.mp4")

output_clip_challenge2 = clip3.fl_image(processImage) #NOTE: this function expects color images!!
%time output_clip_challenge2.write_videofile(harder_challenge_video_output, audio=False)
[MoviePy] >>>> Building video harder_challenge_video_output.mp4
[MoviePy] Writing video harder_challenge_video_output.mp4


  0%|          | 0/1200 [00:00<?, ?it/s]


  0%|          | 1/1200 [00:03<1:11:05,  3.56s/it]


  0%|          | 2/1200 [00:07<1:10:22,  3.53s/it]


  0%|          | 3/1200 [00:10<1:09:47,  3.50s/it]


  0%|          | 4/1200 [00:13<1:09:34,  3.49s/it]


  0%|          | 5/1200 [00:17<1:08:35,  3.44s/it]


  0%|          | 6/1200 [00:20<1:07:59,  3.42s/it]


  1%|          | 7/1200 [00:24<1:08:03,  3.42s/it]


  1%|          | 8/1200 [00:27<1:07:24,  3.39s/it]


  1%|          | 9/1200 [00:30<1:07:35,  3.41s/it]


  1%|          | 10/1200 [00:34<1:07:51,  3.42s/it]


  1%|          | 11/1200 [00:37<1:07:44,  3.42s/it]


  1%|          | 12/1200 [00:41<1:07:30,  3.41s/it]


  1%|          | 13/1200 [00:44<1:07:00,  3.39s/it]


  1%|          | 14/1200 [00:47<1:06:28,  3.36s/it]


  1%|▏         | 15/1200 [00:51<1:06:16,  3.36s/it]


  1%|▏         | 16/1200 [00:54<1:06:19,  3.36s/it]


  1%|▏         | 17/1200 [00:57<1:05:46,  3.34s/it]


  2%|▏         | 18/1200 [01:00<1:05:11,  3.31s/it]


  2%|▏         | 19/1200 [01:04<1:04:59,  3.30s/it]


  2%|▏         | 20/1200 [01:07<1:05:00,  3.31s/it]


  2%|▏         | 21/1200 [01:10<1:04:52,  3.30s/it]


  2%|▏         | 22/1200 [01:14<1:04:04,  3.26s/it]


  2%|▏         | 23/1200 [01:18<1:08:52,  3.51s/it]


  2%|▏         | 24/1200 [01:21<1:07:03,  3.42s/it]


  2%|▏         | 25/1200 [01:24<1:06:14,  3.38s/it]


  2%|▏         | 26/1200 [01:27<1:05:14,  3.33s/it]


  2%|▏         | 27/1200 [01:31<1:05:54,  3.37s/it]


  2%|▏         | 28/1200 [01:34<1:06:31,  3.41s/it]


  2%|▏         | 29/1200 [01:38<1:07:26,  3.46s/it]


  2%|▎         | 30/1200 [01:41<1:08:08,  3.49s/it]


  3%|▎         | 31/1200 [01:46<1:13:13,  3.76s/it]


  3%|▎         | 32/1200 [01:50<1:18:05,  4.01s/it]


  3%|▎         | 33/1200 [01:54<1:16:44,  3.95s/it]


  3%|▎         | 34/1200 [01:58<1:14:05,  3.81s/it]


  3%|▎         | 35/1200 [02:02<1:15:01,  3.86s/it]


  3%|▎         | 36/1200 [02:07<1:22:18,  4.24s/it]


  3%|▎         | 37/1200 [02:11<1:22:03,  4.23s/it]


  3%|▎         | 38/1200 [02:15<1:18:53,  4.07s/it]


  3%|▎         | 39/1200 [02:19<1:17:41,  4.01s/it]


  3%|▎         | 40/1200 [02:23<1:17:19,  4.00s/it]


  3%|▎         | 41/1200 [02:26<1:15:59,  3.93s/it]


  4%|▎         | 42/1200 [02:30<1:13:56,  3.83s/it]


  4%|▎         | 43/1200 [02:33<1:10:58,  3.68s/it]


  4%|▎         | 44/1200 [02:36<1:07:57,  3.53s/it]


  4%|▍         | 45/1200 [02:40<1:05:44,  3.42s/it]


  4%|▍         | 46/1200 [02:43<1:04:02,  3.33s/it]


  4%|▍         | 47/1200 [02:46<1:02:48,  3.27s/it]


  4%|▍         | 48/1200 [02:49<1:01:38,  3.21s/it]


  4%|▍         | 49/1200 [02:52<1:00:58,  3.18s/it]


  4%|▍         | 50/1200 [02:55<1:00:13,  3.14s/it]


  4%|▍         | 51/1200 [02:58<59:35,  3.11s/it]  


  4%|▍         | 52/1200 [03:01<58:50,  3.08s/it]


  4%|▍         | 53/1200 [03:04<58:23,  3.05s/it]


  4%|▍         | 54/1200 [03:07<58:05,  3.04s/it]


  5%|▍         | 55/1200 [03:10<58:32,  3.07s/it]


  5%|▍         | 56/1200 [03:13<58:17,  3.06s/it]


  5%|▍         | 57/1200 [03:16<58:16,  3.06s/it]


  5%|▍         | 58/1200 [03:20<59:47,  3.14s/it]


  5%|▍         | 59/1200 [03:23<1:00:19,  3.17s/it]


  5%|▌         | 60/1200 [03:26<1:00:29,  3.18s/it]


  5%|▌         | 61/1200 [03:29<59:47,  3.15s/it]  


  5%|▌         | 62/1200 [03:33<1:01:01,  3.22s/it]


  5%|▌         | 63/1200 [03:36<1:01:38,  3.25s/it]


  5%|▌         | 64/1200 [03:39<1:02:09,  3.28s/it]


  5%|▌         | 65/1200 [03:42<1:01:34,  3.25s/it]


  6%|▌         | 66/1200 [03:46<1:02:11,  3.29s/it]


  6%|▌         | 67/1200 [03:49<1:02:51,  3.33s/it]


  6%|▌         | 68/1200 [03:52<1:01:57,  3.28s/it]


  6%|▌         | 69/1200 [03:56<1:02:49,  3.33s/it]


  6%|▌         | 70/1200 [03:59<1:02:44,  3.33s/it]


  6%|▌         | 71/1200 [04:03<1:02:44,  3.33s/it]


  6%|▌         | 72/1200 [04:06<1:00:53,  3.24s/it]


  6%|▌         | 73/1200 [04:09<1:01:18,  3.26s/it]


  6%|▌         | 74/1200 [04:12<1:01:01,  3.25s/it]


  6%|▋         | 75/1200 [04:16<1:01:58,  3.31s/it]


  6%|▋         | 76/1200 [04:19<1:02:04,  3.31s/it]


  6%|▋         | 77/1200 [04:22<1:01:37,  3.29s/it]


  6%|▋         | 78/1200 [04:26<1:02:07,  3.32s/it]


  7%|▋         | 79/1200 [04:29<1:03:49,  3.42s/it]


  7%|▋         | 80/1200 [04:33<1:03:49,  3.42s/it]


  7%|▋         | 81/1200 [04:36<1:04:24,  3.45s/it]


  7%|▋         | 82/1200 [04:40<1:06:29,  3.57s/it]


  7%|▋         | 83/1200 [04:44<1:09:31,  3.73s/it]


  7%|▋         | 84/1200 [04:48<1:08:30,  3.68s/it]


  7%|▋         | 85/1200 [04:51<1:08:09,  3.67s/it]


  7%|▋         | 86/1200 [04:56<1:14:40,  4.02s/it]


  7%|▋         | 87/1200 [05:00<1:11:17,  3.84s/it]


  7%|▋         | 88/1200 [05:04<1:13:58,  3.99s/it]


  7%|▋         | 89/1200 [05:08<1:15:54,  4.10s/it]


  8%|▊         | 90/1200 [05:13<1:18:39,  4.25s/it]


  8%|▊         | 91/1200 [05:17<1:17:24,  4.19s/it]


  8%|▊         | 92/1200 [05:21<1:16:25,  4.14s/it]


  8%|▊         | 93/1200 [05:25<1:14:25,  4.03s/it]


  8%|▊         | 94/1200 [05:29<1:13:50,  4.01s/it]


  8%|▊         | 95/1200 [05:32<1:11:13,  3.87s/it]


  8%|▊         | 96/1200 [05:36<1:10:32,  3.83s/it]


  8%|▊         | 97/1200 [05:40<1:09:09,  3.76s/it]


  8%|▊         | 98/1200 [05:44<1:10:26,  3.84s/it]


  8%|▊         | 99/1200 [05:47<1:09:12,  3.77s/it]


  8%|▊         | 100/1200 [05:51<1:08:17,  3.73s/it]


  8%|▊         | 101/1200 [05:54<1:06:15,  3.62s/it]


  8%|▊         | 102/1200 [05:58<1:05:11,  3.56s/it]


  9%|▊         | 103/1200 [06:01<1:05:28,  3.58s/it]


  9%|▊         | 104/1200 [06:05<1:04:46,  3.55s/it]


  9%|▉         | 105/1200 [06:09<1:09:03,  3.78s/it]


  9%|▉         | 106/1200 [06:13<1:10:05,  3.84s/it]


  9%|▉         | 107/1200 [06:17<1:10:09,  3.85s/it]


  9%|▉         | 108/1200 [06:20<1:08:40,  3.77s/it]


  9%|▉         | 109/1200 [06:24<1:07:35,  3.72s/it]


  9%|▉         | 110/1200 [06:28<1:09:37,  3.83s/it]


  9%|▉         | 111/1200 [06:33<1:16:14,  4.20s/it]


  9%|▉         | 112/1200 [06:39<1:24:03,  4.64s/it]


  9%|▉         | 113/1200 [06:44<1:25:36,  4.73s/it]


 10%|▉         | 114/1200 [06:48<1:24:09,  4.65s/it]


 10%|▉         | 115/1200 [06:52<1:20:14,  4.44s/it]


 10%|▉         | 116/1200 [06:56<1:16:27,  4.23s/it]


 10%|▉         | 117/1200 [07:00<1:14:06,  4.11s/it]


 10%|▉         | 118/1200 [07:03<1:11:57,  3.99s/it]


 10%|▉         | 119/1200 [07:07<1:11:37,  3.98s/it]


 10%|█         | 120/1200 [07:11<1:10:42,  3.93s/it]


 10%|█         | 121/1200 [07:15<1:10:45,  3.93s/it]


 10%|█         | 122/1200 [07:20<1:15:24,  4.20s/it]


 10%|█         | 123/1200 [07:24<1:16:33,  4.27s/it]


 10%|█         | 124/1200 [07:28<1:15:31,  4.21s/it]


 10%|█         | 125/1200 [07:33<1:14:30,  4.16s/it]


 10%|█         | 126/1200 [07:36<1:12:48,  4.07s/it]


 11%|█         | 127/1200 [07:40<1:10:48,  3.96s/it]


 11%|█         | 128/1200 [07:44<1:11:51,  4.02s/it]


 11%|█         | 129/1200 [07:48<1:11:32,  4.01s/it]


 11%|█         | 130/1200 [07:52<1:12:06,  4.04s/it]


 11%|█         | 131/1200 [07:56<1:11:54,  4.04s/it]


 11%|█         | 132/1200 [08:00<1:12:11,  4.06s/it]


 11%|█         | 133/1200 [08:05<1:13:48,  4.15s/it]


 11%|█         | 134/1200 [08:09<1:14:05,  4.17s/it]


 11%|█▏        | 135/1200 [08:13<1:15:04,  4.23s/it]


 11%|█▏        | 136/1200 [08:19<1:19:41,  4.49s/it]


 11%|█▏        | 137/1200 [08:25<1:27:56,  4.96s/it]


 12%|█▏        | 138/1200 [08:30<1:29:34,  5.06s/it]


 12%|█▏        | 139/1200 [08:35<1:27:25,  4.94s/it]


 12%|█▏        | 140/1200 [08:39<1:24:34,  4.79s/it]


 12%|█▏        | 141/1200 [08:44<1:23:27,  4.73s/it]


 12%|█▏        | 142/1200 [08:48<1:22:39,  4.69s/it]


 12%|█▏        | 143/1200 [08:53<1:22:58,  4.71s/it]


 12%|█▏        | 144/1200 [08:58<1:22:29,  4.69s/it]


 12%|█▏        | 145/1200 [09:02<1:22:30,  4.69s/it]


 12%|█▏        | 146/1200 [09:07<1:22:58,  4.72s/it]


 12%|█▏        | 147/1200 [09:12<1:23:04,  4.73s/it]


 12%|█▏        | 148/1200 [09:17<1:27:33,  4.99s/it]


 12%|█▏        | 149/1200 [09:23<1:30:54,  5.19s/it]


 12%|█▎        | 150/1200 [09:28<1:29:56,  5.14s/it]


 13%|█▎        | 151/1200 [09:33<1:29:22,  5.11s/it]


 13%|█▎        | 152/1200 [09:38<1:29:25,  5.12s/it]


 13%|█▎        | 153/1200 [09:44<1:30:20,  5.18s/it]


 13%|█▎        | 154/1200 [09:49<1:30:20,  5.18s/it]


 13%|█▎        | 155/1200 [09:54<1:27:56,  5.05s/it]


 13%|█▎        | 156/1200 [09:58<1:25:52,  4.94s/it]
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-26-e16b98e04956> in <module>()
      3 
      4 output_clip_challenge2 = clip3.fl_image(processImage) #NOTE: this function expects color images!!
----> 5 get_ipython().magic('time output_clip_challenge2.write_videofile(harder_challenge_video_output, audio=False)')

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/IPython/core/interactiveshell.py in magic(self, arg_s)
   2156         magic_name, _, magic_arg_s = arg_s.partition(' ')
   2157         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
-> 2158         return self.run_line_magic(magic_name, magic_arg_s)
   2159 
   2160     #-------------------------------------------------------------------------

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/IPython/core/interactiveshell.py in run_line_magic(self, magic_name, line)
   2077                 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
   2078             with self.builtin_trap:
-> 2079                 result = fn(*args,**kwargs)
   2080             return result
   2081 

<decorator-gen-59> in time(self, line, cell, local_ns)

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/IPython/core/magic.py in <lambda>(f, *a, **k)
    186     # but it's overkill for just that one bit of state.
    187     def magic_deco(arg):
--> 188         call = lambda f, *a, **k: f(*a, **k)
    189 
    190         if callable(arg):

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/IPython/core/magics/execution.py in time(self, line, cell, local_ns)
   1174         if mode=='eval':
   1175             st = clock2()
-> 1176             out = eval(code, glob, local_ns)
   1177             end = clock2()
   1178         else:

<timed eval> in <module>()

<decorator-gen-172> in write_videofile(self, filename, fps, codec, bitrate, audio, audio_fps, preset, audio_nbytes, audio_codec, audio_bitrate, audio_bufsize, temp_audiofile, rewrite_audio, remove_temp, write_logfile, verbose, threads, ffmpeg_params)

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/moviepy/decorators.py in requires_duration(f, clip, *a, **k)
     52         raise ValueError("Attribute 'duration' not set")
     53     else:
---> 54         return f(clip, *a, **k)
     55 
     56 

<decorator-gen-171> in write_videofile(self, filename, fps, codec, bitrate, audio, audio_fps, preset, audio_nbytes, audio_codec, audio_bitrate, audio_bufsize, temp_audiofile, rewrite_audio, remove_temp, write_logfile, verbose, threads, ffmpeg_params)

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/moviepy/decorators.py in use_clip_fps_by_default(f, clip, *a, **k)
    135              for (k,v) in k.items()}
    136 
--> 137     return f(clip, *new_a, **new_kw)

<decorator-gen-170> in write_videofile(self, filename, fps, codec, bitrate, audio, audio_fps, preset, audio_nbytes, audio_codec, audio_bitrate, audio_bufsize, temp_audiofile, rewrite_audio, remove_temp, write_logfile, verbose, threads, ffmpeg_params)

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/moviepy/decorators.py in convert_masks_to_RGB(f, clip, *a, **k)
     20     if clip.ismask:
     21         clip = clip.to_RGB()
---> 22     return f(clip, *a, **k)
     23 
     24 @decorator.decorator

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/moviepy/video/VideoClip.py in write_videofile(self, filename, fps, codec, bitrate, audio, audio_fps, preset, audio_nbytes, audio_codec, audio_bitrate, audio_bufsize, temp_audiofile, rewrite_audio, remove_temp, write_logfile, verbose, threads, ffmpeg_params)
    337                            audiofile = audiofile,
    338                            verbose=verbose, threads=threads,
--> 339                            ffmpeg_params=ffmpeg_params)
    340 
    341         if remove_temp and make_audio:

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/moviepy/video/io/ffmpeg_writer.py in ffmpeg_write_video(clip, filename, fps, codec, bitrate, preset, withmask, write_logfile, audiofile, verbose, threads, ffmpeg_params)
    202 
    203     for t,frame in clip.iter_frames(progress_bar=True, with_times=True,
--> 204                                     fps=fps, dtype="uint8"):
    205         if withmask:
    206             mask = (255*clip.mask.get_frame(t))

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/tqdm/_tqdm.py in __iter__(self)
    828 """, fp_write=getattr(self.fp, 'write', sys.stderr.write))
    829 
--> 830             for obj in iterable:
    831                 yield obj
    832                 # Update and print the progressbar.

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/moviepy/Clip.py in generator()
    471             for t in np.arange(0, self.duration, 1.0/fps):
    472 
--> 473                 frame = self.get_frame(t)
    474 
    475                 if (dtype is not None) and (frame.dtype != dtype):

<decorator-gen-135> in get_frame(self, t)

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/moviepy/decorators.py in wrapper(f, *a, **kw)
     87         new_kw = {k: fun(v) if k in varnames else v
     88                  for (k,v) in kw.items()}
---> 89         return f(*new_a, **new_kw)
     90     return decorator.decorator(wrapper)
     91 

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/moviepy/Clip.py in get_frame(self, t)
     93                 return frame
     94         else:
---> 95             return self.make_frame(t)
     96 
     97     def fl(self, fun, apply_to=[] , keep_duration=True):

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/moviepy/Clip.py in <lambda>(t)
    134 
    135         #mf = copy(self.make_frame)
--> 136         newclip = self.set_make_frame(lambda t: fun(self.get_frame, t))
    137 
    138         if not keep_duration:

/Users/tmcracken/anaconda/envs/carnd-term1/lib/python3.5/site-packages/moviepy/video/VideoClip.py in <lambda>(gf, t)
    512         `get_frame(t)` by another frame,  `image_func(get_frame(t))`
    513         """
--> 514         return self.fl(lambda gf, t: image_func(gf(t)), apply_to)
    515 
    516     # --------------------------------------------------------------

<ipython-input-19-3511ec1d3cab> in processImage(img)
     12     binary_top_down = myPipeline(top_down)
     13     #4. Detect the lines and fit polynomial
---> 14     leftline = detectLines(binary_top_down, theLeftLine, 'left')
     15     rightline = detectLines(binary_top_down, theRightLine, 'right')
     16     lines = {'left_line': leftline, 'right_line': rightline }

<ipython-input-25-c2cebb0f14a6> in detectLines(img, line, direction)
     55             for yval in range(int(img.shape[0]*((winHeight-i-1)/winHeight)), int(img.shape[0]*((winHeight-i)/winHeight))):
     56                 for xval in range(peak-winWidth, peak+winWidth):
---> 57                     if img[yval][xval] == 1.0:
     58                         yvals.append(yval)
     59                         xvals.append(xval)

IndexError: index 1280 is out of bounds for axis 0 with size 1280
In [360]:
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(harder_challenge_video_output))
Out[360]:
In [ ]: